card_id=$1

dev1=`expr ${card_id} \* 4`
dev2=`expr $dev1 + 1`
dev3=`expr $dev1 + 2`
dev4=`expr $dev1 + 3`

master_port=`expr $dev2 + 50000`


echo $dev1 
echo $dev2 
echo $dev3 
echo $dev4

CURTIME=`date +%m%d%H%M`

sn=`speclk $dev1 3 | awk '{print $5}'`

ngcfile=ngc_$sn
if [ -f $ngcfile ];then
    rm $ngcfile
fi
touch $ngcfile

export TECO_LAUNCH_BLOCKING=1

SDAA_VISIBLE_DEVICES=$dev1,$dev2,$dev3,$dev4 \
SDAA_VISIBLE_DEVICES=$dev1,$dev2,$dev3,$dev4 python3 run_ngc_resnet.py --workspace logs_$sn --master_port $dev2 --model_name resnet50 --nproc_per_node 4 --bs 64 --lr 0.256 --device swai --epoch 3 --dataset_path /home/mini_image_classification_1000  --grad_scale True --autocast True | tee -a $ngcfile &

wait


sleep 5

result=$(cat $ngcfile | grep -c 'Summary:')
length=$(expr length "$result")
echo "result is $result"

if [[ $result -ne 2 ]];
then
        echo -e "NGC CARD  $1 TEST RESULT: \E[31m [FAILED] \E[0m" | tee -a $ngcfile
else
        echo -e "NGC CARD $1 TEST RESULT: \E[32m [PASS] \E[0m" | tee -a $ngcfile
fi
